package ner;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashSet;

import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;

public class KeyGenerator  extends DefaultHandler {

	private String[] skipList = {"s", "p", "TXT", "DOC", "DOCID", "DOCNO", "HL", "DD", "SO", "IN", "DATELINE", };
	private HashSet<String> skipSet = new HashSet<String>();

	public static void main(String[] args) {
		String keysFile = "D:/study/natural language processing/Assignments/Sequential tagging/NE data/ne_files/ne.muc6.test.keys";
		String outputFile = "D:/study/natural language processing/Assignments/Sequential tagging/NE data/ne_files/ne.muc6.test.output.keys";
		String interFile = "D:/study/natural language processing/Assignments/Sequential tagging/NE data/ne_files/ne.muc6.test.inter.keys";
		
		KeyGenerator generator = new KeyGenerator();
		generator.generateKeys(keysFile, outputFile, interFile);
	}

	private String inputFile;
	private String outputFile;
	private BufferedWriter writer;

	public void generateKeys(String keyFile, String outputFile, String interFile) {
		try {
			this.inputFile = keyFile;
			this.outputFile = outputFile;
			this.writer = new BufferedWriter(new FileWriter(this.outputFile));

			for (String skipWord : skipList) {
				skipSet.add(skipWord);
			}
			
			FileWriter writer = new FileWriter(interFile);
			FileReader reader = new FileReader(inputFile);

			XMLReader xmlr = XMLReaderFactory.createXMLReader();

			xmlr.setContentHandler(this);
			xmlr.setErrorHandler(this);

			BufferedReader br = new BufferedReader(reader);

			try {
				format(br, writer);
				reader.close();
			}
			catch (IOException ex) {
				throw new RuntimeException(ex);
			}

			reader = new FileReader(interFile);

			// Parse the incoming XML file.
			xmlr.parse(new InputSource(reader));
		}
		catch (IOException ex) {
			throw new RuntimeException(ex);
		}
		catch (SAXException e) {
			throw new RuntimeException(e);
		}

	}

	public static void format(BufferedReader br, FileWriter out)
	throws IOException
	{
		String line;

		try {
			out.write("<ROOT>" + "\n");
			while ((line = br.readLine()) != null) {
				line = line.replaceAll("&", "@amp;");
				out.write(line.trim() + "\n");
			}
			out.write("</ROOT>" + "\n");
		}
		catch (IOException ex) {
			System.err.println(ex);
		}

		out.close();
		br.close();
	}

	boolean skip = true;
	String tag;

	@Override
	public void startElement(String uri, String localName, String qName,
			Attributes atts) throws SAXException {

		if(skipSet.contains(qName)) {
			skip = true;
		}
		else {
			skip = false;
			tag = atts.getValue("TYPE");
		}
	}

	@Override
	public void endElement(String uri, String localName, String qName)
	throws SAXException {
		// TODO Auto-generated method stub
		super.endElement(uri, localName, qName);
	}

	@Override
	public void characters(char[] ch, int start, int length)
	throws SAXException {
		try {
			String text = new String(ch, start, length);

			String uText = unescapeText(text);
			if(skip) {
				uText = uText.replaceAll(",", "\n,").replaceAll("'s", "\n's");
			}
			uText = uText.replaceAll("[(]", "(\n");

			if (skip) {
				String[] txts = uText.split(" |[.]|\\n");
				for (String txt : txts) {
					txt = txt.trim();
					if (txt != null && !txt.isEmpty() && !txt.equals("\n")) {
						writer.write(Training.NONER + " " + txt + "\n");
					}
				}
			}
			else {
				String[] txts = uText.split(" |[.]|\\n");
				for (String txt : txts) {
					if (txt != null && !txt.isEmpty() && !txt.equals("\n")) {
						writer.write(tag + " " + txt + "\n");
					}
				}
				skip = true;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	@Override
	public void endDocument() throws SAXException {
		// TODO Auto-generated method stub
		super.endDocument();
		try {
			writer.flush();
			writer.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	public String unescapeText(String text)
	{
		text = text.replaceAll("@amp;", "&");
		text = text.replaceAll("&MD;", "-");
		text = text.replaceAll("&AMP;", "&");
		text = text.replaceAll("&LR;", "");

		text = text.replaceAll("``|''", "\"");
		return text;
	}
}
